'''
Created on 2012-2-6

@author: xiaokan
'''
from xiaokan import robotexclusionrulesparser
import urlparse

class RobotsHelper(object):
    
    '''
    This class is a singleton class, no matter how much times you use RobotsHelper() to new an instance, you always get that unique one.
    '''

    #USER AGENT
    USER_AGENT = "Mozilla/5.0"
    _instance = None

    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(RobotsHelper, cls).__new__(
                                cls, *args, **kwargs)
        return cls._instance

    def is_url_allowed(self, url):
        
        '''
        Since this class is a singleton, the initial of base_map could 
        not be placed in __init__ that could cause base_map initiate 
        many times, so here uses a little tricky to test the existence 
        of it and create it if is doesn't exist which will raise an error but caught by this try..except
        '''
        
        try:                
            self.base_map["TEST_THE_EXISTENCE_OF_BASE_MAP"] = None
        except:
            self.base_map = {}
            
        '''Start here'''

        try:
            result = urlparse.urlparse(url)
            base_url = result.netloc
            path = result.path
            if path.find("cgi") != -1:
                return False
            scheme = result.scheme
            
            if not self.base_map.has_key(base_url): #Robots.txt in this base address haven't been parsed before, parse it and save the parser in dictionary
                
                print "Robots: ", base_url
                rerp = robotexclusionrulesparser.RobotExclusionRulesParser()
                rerp.fetch(scheme + "://" + base_url + "/robots.txt")
                self.base_map[base_url] = rerp            
            else:   #Otherwise, get it from dictionary and check if it is allowed
                rerp = self.base_map[base_url]
                
            return rerp.is_allowed(self.USER_AGENT, path)
        except Exception:
            return True
        
            
        
